{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 数据预处理"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import matplotlib.pyplot as plt\n",
    "import seaborn as sns\n",
    "import numpy as np\n",
    "from scipy.stats import norm\n",
    "from sklearn.preprocessing import StandardScaler\n",
    "from scipy import stats\n",
    "df_train = pd.read_csv('../data/train.csv')\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Total</th>\n",
       "      <th>Percent</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>PoolQC</th>\n",
       "      <td>1453</td>\n",
       "      <td>0.995205</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>MiscFeature</th>\n",
       "      <td>1406</td>\n",
       "      <td>0.963014</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Alley</th>\n",
       "      <td>1369</td>\n",
       "      <td>0.937671</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Fence</th>\n",
       "      <td>1179</td>\n",
       "      <td>0.807534</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>FireplaceQu</th>\n",
       "      <td>690</td>\n",
       "      <td>0.472603</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>LotFrontage</th>\n",
       "      <td>259</td>\n",
       "      <td>0.177397</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>GarageYrBlt</th>\n",
       "      <td>81</td>\n",
       "      <td>0.055479</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>GarageCond</th>\n",
       "      <td>81</td>\n",
       "      <td>0.055479</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>GarageType</th>\n",
       "      <td>81</td>\n",
       "      <td>0.055479</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>GarageFinish</th>\n",
       "      <td>81</td>\n",
       "      <td>0.055479</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>GarageQual</th>\n",
       "      <td>81</td>\n",
       "      <td>0.055479</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>BsmtFinType2</th>\n",
       "      <td>38</td>\n",
       "      <td>0.026027</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>BsmtExposure</th>\n",
       "      <td>38</td>\n",
       "      <td>0.026027</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>BsmtQual</th>\n",
       "      <td>37</td>\n",
       "      <td>0.025342</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>BsmtCond</th>\n",
       "      <td>37</td>\n",
       "      <td>0.025342</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>BsmtFinType1</th>\n",
       "      <td>37</td>\n",
       "      <td>0.025342</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>MasVnrArea</th>\n",
       "      <td>8</td>\n",
       "      <td>0.005479</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>MasVnrType</th>\n",
       "      <td>8</td>\n",
       "      <td>0.005479</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Electrical</th>\n",
       "      <td>1</td>\n",
       "      <td>0.000685</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Id</th>\n",
       "      <td>0</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "              Total   Percent\n",
       "PoolQC         1453  0.995205\n",
       "MiscFeature    1406  0.963014\n",
       "Alley          1369  0.937671\n",
       "Fence          1179  0.807534\n",
       "FireplaceQu     690  0.472603\n",
       "LotFrontage     259  0.177397\n",
       "GarageYrBlt      81  0.055479\n",
       "GarageCond       81  0.055479\n",
       "GarageType       81  0.055479\n",
       "GarageFinish     81  0.055479\n",
       "GarageQual       81  0.055479\n",
       "BsmtFinType2     38  0.026027\n",
       "BsmtExposure     38  0.026027\n",
       "BsmtQual         37  0.025342\n",
       "BsmtCond         37  0.025342\n",
       "BsmtFinType1     37  0.025342\n",
       "MasVnrArea        8  0.005479\n",
       "MasVnrType        8  0.005479\n",
       "Electrical        1  0.000685\n",
       "Id                0  0.000000"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# 统计缺失数据\n",
    "total = df_train.isnull().sum().sort_values(ascending=False)\n",
    "percent = (df_train.isnull().sum()/df_train.isnull().count()).sort_values(ascending=False)\n",
    "missing_data = pd.concat([total, percent], axis=1, keys=['Total', 'Percent'])\n",
    "missing_data.head(20)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 如何处理缺失的数据\n",
    "\n",
    "超过15%的数据缺失时，我们应该删除相应的变量，假装它从未存在过。\n",
    "这意味着，在这些情况下，我们不会尝试任何技巧来填补缺失的数据。所以，有一些变量（如'PoolQC'、'MiscFeature'、'Alley'等）我们应该删除。\n",
    "我们会错过这些数据吗？我认为不会。这些变量似乎都不是很重要，因为它们大多数都不是我们在买房时考虑的方面（也许这就是数据丢失的原因？） 此外，仔细观察这些变量，我们可以说像 \"PoolQC\"、\"MiscFeature \"和 \"FireplaceQu \"这样的变量是离群值的有力候选者，所以我们会很乐意删除它们。\n",
    "\n",
    "在涉及其余的情况下，我们可以看到'GarageX'变量有相同数量的缺失数据。我打赌缺失的数据是指同一组观察值（尽管我不会检查它；这只是5%，我们不应该在5美元的问题上花费20美元）。由于关于车库的最重要信息是由'GarageCars'表达的，考虑到我们只是在讨论5%的缺失数据，我将删除提到的'GarageX'变量。同样的逻辑也适用于'BsmtX'变量。\n",
    "\n",
    "关于'MasVnrArea'和'MasVnrType'，我们可以认为这些变量并不重要，它们与已经考虑过的'YearBuilt'和'OverallQual'有很强的关联性。所以，如果我们删除'MasVnrArea'和'MasVnrType'，就不会丢失信息。\n",
    "\n",
    "最后，我们在 \"Electrical\"中发现了一个缺失的观察值。因为它只是一个观察值，所以我们将删除这个观察值并保留这个变量。\n",
    "\n",
    "综上所述，为了处理缺失数据，我们将删除所有缺失数据的变量，除了'Electrical'这个变量。在'Electrical'中，我们将只删除有缺失数据的观察值。"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\jiang\\AppData\\Local\\Temp/ipykernel_1416/41399952.py:1: FutureWarning: In a future version of pandas all arguments of DataFrame.drop except for the argument 'labels' will be keyword-only\n",
      "  df_train = df_train.drop((missing_data[missing_data['Total'] > 1]).index,1)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "0"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_train = df_train.drop((missing_data[missing_data['Total'] > 1]).index, 1)\n",
    "df_train = df_train.drop(df_train.loc[df_train['Electrical'].isnull()].index)\n",
    "df_train.isnull().sum().max()  # 没有缺失值了\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_train.to_csv('../data/train_pp.csv',index = False)"
   ]
  }
 ],
 "metadata": {
  "interpreter": {
   "hash": "a9661a0f83c933cf25e0dd40af67627cb20747a37143c65b775945a664681eb4"
  },
  "kernelspec": {
   "display_name": "Python 3.8.5 64-bit",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.5"
  },
  "orig_nbformat": 4
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
